# encoding: utf-8
# author: walker
# date:
# summary:

import os
import sys
import time
from configparser import ConfigParser

from pyhdfs import HdfsClient

cur_dir_fullpath = os.path.dirname(os.path.abspath(__file__))
StartTime = time.time()
FileSize = 0  # 文件总大小

LocalDir = ''
HdfsDir = ''
NameNode = ''


# 读取配置文件
def ReadConfig():
    global LocalDir, HdfsDir, NameNode

    cfg = ConfigParser()
    cfgFile = os.path.join(cur_dir_fullpath, 'config.ini')
    if not os.path.exists(cfgFile):
        input(cfgFile + ' not found')
        sys.exit(-1)
    cfgLst = cfg.read(cfgFile)
    if len(cfgLst) < 1:
        input('Read config.ini failed...')
        sys.exit(-1)

    LocalDir = cfg.get('config', 'LocalDir').strip()
    if not os.path.exists(LocalDir):
        input(LocalDir + ' not found')
        sys.exit(-1)
    print('LocalDir:' + LocalDir)

    HdfsDir = cfg.get('config', 'HdfsDir').strip()
    print('HdfsDir:' + HdfsDir)

    NameNode = cfg.get('config', 'NameNode').strip()
    print('NameNode:' + NameNode)

    print('Read config.ini successed!')


# 处理一个
def ProcOne(client, srcFile, dstFile):
    global FileSize
    print('ProcOne \n%s\n -> \n%s ' % (srcFile, dstFile))

    # 目标文件已经存在且大小相同
    if client.exists(dstFile) and \
            (os.path.getsize(srcFile) == client.list_status(dstFile)[0].length):
        print('file exists: %s ' % dstFile)
        return True

    # 注意，如果已存在会被覆盖
    client.copy_from_local(srcFile, dstFile, overwrite=True)

    if os.path.getsize(srcFile) == client.list_status(dstFile)[0].length:  # 校验文件大小
        FileSize += os.path.getsize(srcFile)
        return True

    return False


# 处理所有
def ProcAll():
    client = HdfsClient(hosts=NameNode, user_name="xujiang")
    if not client.exists(HdfsDir):
        print(HdfsDir + ' not found')
        sys.exit(-1)
    total = len(os.listdir(LocalDir))
    processed = 0
    failedList = list()
    for filename in os.listdir(LocalDir):
        srcFile = os.path.join(LocalDir, filename)
        dstFile = HdfsDir + '/' + filename
        if not ProcOne(client, srcFile, dstFile):
            failedList.append(srcFile)
        processed += 1
        print('%d/%d/%d, time cost: %.2f s' % (total, processed, len(failedList), time.time() - StartTime))
        print('%d B, %.2f MB/s \n' % (FileSize, FileSize / 1024 / 1024 / (time.time() - StartTime)))

    if failedList:
        print('failedList: %s' % repr(failedList))
    else:
        print('Good! No Error!')
        print('%d B, %.2f MB/s' % (FileSize, FileSize / 1024 / 1024 / (time.time() - StartTime)))


if __name__ == '__main__':
    ReadConfig()
    ProcAll()
    print('Time total: %.2f s' % (time.time() - StartTime))
    print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
